from urllib.robotparser import RobotFileParser
from urllib.request import urlopen

robotTxt = "http://www.jianshu.com/robots.txt"
#rp = RobotFileParser(robotTxt)
#rp.read()
rp = RobotFileParser()
rp.set_url(robotTxt)
rp.read()
#rp.parse(urlopen(robotTxt).read().decode('utf-8').split('\n'))
#不应该全都是False，不知哪有问题，初步怀疑是因为权限或者请求被拒绝了（读取robots.txt)
print(rp.can_fetch("*", "http://www.jianshu.com/p/b67554025d7d"))
print(rp.can_fetch("*", "http://www.jianshu.com/search?q=python&page=1&type=collections"))
